{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluation of oversamplers with a set of classifiers on a set of datasets\n",
    "\n",
    "In this notebook, we give an example of evaluating multiple oversamplers on multiple datasets with multiple classifiers. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os.path\n",
    "\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "\n",
    "import smote_variants as sv\n",
    "\n",
    "import imbalanced_databases as imbd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Setting the cache_path which is used for caching during the evaluation\n",
    "\n",
    "cache_path= os.path.join(os.path.expanduser('~'), 'smote_test')\n",
    "\n",
    "if not os.path.exists(cache_path):\n",
    "    os.makedirs(cache_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Specifying two datasets by their load functions\n",
    "\n",
    "datasets= [imbd.load_glass0, imbd.load_yeast1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Specifying the classifiers used for evaluation\n",
    "\n",
    "knn_classifier= KNeighborsClassifier()\n",
    "dt_classifier= DecisionTreeClassifier()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-06-11 18:18:29,340:INFO:dataset: glass0, samplings_available: True, evaluations_available: True\n",
      "2019-06-11 18:18:29,341:INFO:doing the folding\n",
      "2019-06-11 18:18:29,341:INFO:Folding reading from file folding_glass0.pickle\n",
      "2019-06-11 18:18:29,344:INFO:do the samplings\n",
      "2019-06-11 18:18:29,345:INFO:create sampling objects\n",
      "2019-06-11 18:18:29,346:INFO:executing 72 sampling in parallel\n",
      "2019-06-11 18:18:31,552:INFO:do the evaluations\n",
      "2019-06-11 18:18:31,552:INFO:create classifier jobs\n",
      "2019-06-11 18:18:31,593:INFO:executing 72 evaluation jobs in parallel\n",
      "2019-06-11 18:18:32,193:INFO:concatenating the results\n",
      "2019-06-11 18:18:32,423:INFO:aggregating the results\n",
      "2019-06-11 18:18:32,773:INFO:dataset: yeast1, samplings_available: True, evaluations_available: True\n",
      "2019-06-11 18:18:32,773:INFO:doing the folding\n",
      "2019-06-11 18:18:32,793:INFO:Folding reading from file folding_yeast1.pickle\n",
      "2019-06-11 18:18:33,028:INFO:do the samplings\n",
      "2019-06-11 18:18:33,029:INFO:create sampling objects\n",
      "2019-06-11 18:18:33,030:INFO:executing 72 sampling in parallel\n",
      "2019-06-11 18:18:33,592:INFO:do the evaluations\n",
      "2019-06-11 18:18:33,592:INFO:create classifier jobs\n",
      "2019-06-11 18:18:33,618:INFO:executing 72 evaluation jobs in parallel\n",
      "2019-06-11 18:18:34,161:INFO:concatenating the results\n",
      "2019-06-11 18:18:34,357:INFO:aggregating the results\n"
     ]
    }
   ],
   "source": [
    "# Executing the evaluation using 5 parallel jobs, and at most 35 different \n",
    "# random but meaningful parameter combinations with the oversamplers\n",
    "\n",
    "results= sv.evaluate_oversamplers(datasets= datasets,\n",
    "                                    samplers= sv.get_n_quickest_oversamplers(5),\n",
    "                                    classifiers= [knn_classifier, dt_classifier],\n",
    "                                    cache_path= cache_path,\n",
    "                                    n_jobs= 5,\n",
    "                                    max_samp_par_comb= 35)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['db_name', 'classifier', 'sampler', 'auc', 'brier', 'acc', 'f1',\n",
      "       'p_top20', 'gacc', 'runtime', 'db_size', 'db_n_attr',\n",
      "       'imbalanced_ratio', 'sampler_categories', 'classifier_parameters_auc',\n",
      "       'classifier_parameters_acc', 'classifier_parameters_gacc',\n",
      "       'classifier_parameters_f1', 'classifier_parameters_p_top20',\n",
      "       'classifier_parameters_brier', 'sampler_parameters_auc',\n",
      "       'sampler_parameters_acc', 'sampler_parameters_gacc',\n",
      "       'sampler_parameters_f1', 'sampler_parameters_p_top20',\n",
      "       'sampler_parameters_brier'],\n",
      "      dtype='object')\n"
     ]
    }
   ],
   "source": [
    "# The results are arranged in a pandas DataFrame with the following columns:\n",
    "# db_name - name of the database\n",
    "# classifier - name of the classifier\n",
    "# sampler - name of the oversampling technique\n",
    "# auc - highest auc score with the classifier and oversampler (aggregated over all classifier and oversampler\n",
    "# parameter combinations)\n",
    "# brier - highest brier score with the classifier and oversampler (aggregated similarly)\n",
    "# acc - the highest accuracy score with the classifier and oversampler (aggregated similarly)\n",
    "# f1 - the highest f1 score with the classifier and oversampler (aggregated similarly)\n",
    "# p_top20 - the highest p_top20 score with the classifier and oversampler (aggregated similarly)\n",
    "# gacc - the highest GACC score with the classifier and oversampler (aggregated similarly)\n",
    "# runtime - average runtime in seconds\n",
    "# db_size - size of the dataset\n",
    "# db_n_attr - number of attributes in the dataset\n",
    "# imbalanced_ratio - the ratio of majority/minority class sizes\n",
    "# sampler_categories - the categories assigned to the oversampler\n",
    "# classifier_parameters_auc - the classifier parameters reaching the highest auc score\n",
    "# classifier_parameters_acc - the classifier parameters reaching the highest acc score\n",
    "# classifier_parameters_gacc - the classifier parameters reaching the highest gacc score\n",
    "# classifier_parameters_f1 - the classifier parameters reaching the highest f1 score\n",
    "# classifier_parameters_p_top20 - the classifier parameters reaching the highest p_top20 score\n",
    "# classifier_parameters_brier - the classifier parameters reaching the highest brier score\n",
    "# sampler_parameters_auc - the oversampler parameters reaching the highest auc score\n",
    "# sampler_parameters_acc - the oversampler parameters reaching the highest acc score\n",
    "# sampler_parameters_gacc - the oversampler parameters reaching the highest gacc score\n",
    "# sampler_parameters_f1 - the oversampler parameters reaching the highest f1 score\n",
    "# sampler_parameters_p_top20 - the oversampler parameters reaching the highest p_top20 score\n",
    "# sampler_parameters_brier - the oversampler parameters reaching the highest brier score\n",
    "\n",
    "print(results.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   db_name              classifier   sampler       auc     brier       acc  \\\n",
      "0   glass0  DecisionTreeClassifier    Gazzah  0.497751  0.331776  0.668224   \n",
      "1   glass0  DecisionTreeClassifier  NT_SMOTE  0.789187  0.186916  0.813084   \n",
      "2   glass0  DecisionTreeClassifier      OUPS  0.795106  0.186916  0.813084   \n",
      "3   glass0  DecisionTreeClassifier   SMOTE_D  0.815377  0.176012  0.823988   \n",
      "4   glass0  DecisionTreeClassifier       SPY  0.786012  0.213396  0.786604   \n",
      "5   glass0    KNeighborsClassifier    Gazzah  0.864253  0.149097  0.805296   \n",
      "6   glass0    KNeighborsClassifier  NT_SMOTE  0.865653  0.156075  0.788162   \n",
      "7   glass0    KNeighborsClassifier      OUPS  0.870662  0.154019  0.795950   \n",
      "8   glass0    KNeighborsClassifier   SMOTE_D  0.877403  0.155389  0.789720   \n",
      "9   glass0    KNeighborsClassifier       SPY  0.867907  0.152960  0.788162   \n",
      "10  yeast1  DecisionTreeClassifier    Gazzah  0.508739  0.286164  0.713836   \n",
      "11  yeast1  DecisionTreeClassifier  NT_SMOTE  0.666199  0.285490  0.714510   \n",
      "12  yeast1  DecisionTreeClassifier      OUPS  0.669999  0.288634  0.711366   \n",
      "13  yeast1  DecisionTreeClassifier   SMOTE_D  0.674151  0.290431  0.709569   \n",
      "14  yeast1  DecisionTreeClassifier       SPY  0.677967  0.286613  0.713387   \n",
      "15  yeast1    KNeighborsClassifier    Gazzah  0.768828  0.186667  0.758086   \n",
      "16  yeast1    KNeighborsClassifier  NT_SMOTE  0.744450  0.185130  0.738095   \n",
      "17  yeast1    KNeighborsClassifier      OUPS  0.766721  0.180180  0.755615   \n",
      "18  yeast1    KNeighborsClassifier   SMOTE_D  0.744332  0.183172  0.743261   \n",
      "19  yeast1    KNeighborsClassifier       SPY  0.762945  0.182417  0.742588   \n",
      "\n",
      "          f1   p_top20      gacc   runtime  \\\n",
      "0   0.009302  0.335938  0.068686  0.015744   \n",
      "1   0.715640  0.750000  0.787531  0.009929   \n",
      "2   0.721839  0.726562  0.793687  0.197963   \n",
      "3   0.746067  0.781250  0.814997  0.015473   \n",
      "4   0.702820  0.695312  0.783954  0.019324   \n",
      "5   0.734607  0.835938  0.809936  0.015744   \n",
      "6   0.714286  0.843750  0.793492  0.009929   \n",
      "7   0.723577  0.828125  0.802224  0.197963   \n",
      "8   0.731141  0.843750  0.808806  0.015473   \n",
      "9   0.716667  0.804688  0.795767  0.019324   \n",
      "10  0.049671  0.288764  0.161299  0.080508   \n",
      "11  0.527685  0.512360  0.656280  0.071769   \n",
      "12  0.538108  0.511236  0.669154  0.159048   \n",
      "13  0.540368  0.508989  0.669231  0.111611   \n",
      "14  0.548561  0.506742  0.677734  0.169086   \n",
      "15  0.584377  0.619101  0.701951  0.080508   \n",
      "16  0.555867  0.568539  0.683737  0.071769   \n",
      "17  0.588663  0.628090  0.706485  0.159048   \n",
      "18  0.556565  0.578652  0.684461  0.111611   \n",
      "19  0.574518  0.611236  0.696984  0.169086   \n",
      "\n",
      "                     ...                    \\\n",
      "0                    ...                     \n",
      "1                    ...                     \n",
      "2                    ...                     \n",
      "3                    ...                     \n",
      "4                    ...                     \n",
      "5                    ...                     \n",
      "6                    ...                     \n",
      "7                    ...                     \n",
      "8                    ...                     \n",
      "9                    ...                     \n",
      "10                   ...                     \n",
      "11                   ...                     \n",
      "12                   ...                     \n",
      "13                   ...                     \n",
      "14                   ...                     \n",
      "15                   ...                     \n",
      "16                   ...                     \n",
      "17                   ...                     \n",
      "18                   ...                     \n",
      "19                   ...                     \n",
      "\n",
      "                           classifier_parameters_gacc  \\\n",
      "0   {'class_weight': None, 'criterion': 'gini', 'm...   \n",
      "1   {'class_weight': None, 'criterion': 'gini', 'm...   \n",
      "2   {'class_weight': None, 'criterion': 'gini', 'm...   \n",
      "3   {'class_weight': None, 'criterion': 'gini', 'm...   \n",
      "4   {'class_weight': None, 'criterion': 'gini', 'm...   \n",
      "5   {'algorithm': 'auto', 'leaf_size': 30, 'metric...   \n",
      "6   {'algorithm': 'auto', 'leaf_size': 30, 'metric...   \n",
      "7   {'algorithm': 'auto', 'leaf_size': 30, 'metric...   \n",
      "8   {'algorithm': 'auto', 'leaf_size': 30, 'metric...   \n",
      "9   {'algorithm': 'auto', 'leaf_size': 30, 'metric...   \n",
      "10  {'class_weight': None, 'criterion': 'gini', 'm...   \n",
      "11  {'class_weight': None, 'criterion': 'gini', 'm...   \n",
      "12  {'class_weight': None, 'criterion': 'gini', 'm...   \n",
      "13  {'class_weight': None, 'criterion': 'gini', 'm...   \n",
      "14  {'class_weight': None, 'criterion': 'gini', 'm...   \n",
      "15  {'algorithm': 'auto', 'leaf_size': 30, 'metric...   \n",
      "16  {'algorithm': 'auto', 'leaf_size': 30, 'metric...   \n",
      "17  {'algorithm': 'auto', 'leaf_size': 30, 'metric...   \n",
      "18  {'algorithm': 'auto', 'leaf_size': 30, 'metric...   \n",
      "19  {'algorithm': 'auto', 'leaf_size': 30, 'metric...   \n",
      "\n",
      "                             classifier_parameters_f1  \\\n",
      "0   {'class_weight': None, 'criterion': 'gini', 'm...   \n",
      "1   {'class_weight': None, 'criterion': 'gini', 'm...   \n",
      "2   {'class_weight': None, 'criterion': 'gini', 'm...   \n",
      "3   {'class_weight': None, 'criterion': 'gini', 'm...   \n",
      "4   {'class_weight': None, 'criterion': 'gini', 'm...   \n",
      "5   {'algorithm': 'auto', 'leaf_size': 30, 'metric...   \n",
      "6   {'algorithm': 'auto', 'leaf_size': 30, 'metric...   \n",
      "7   {'algorithm': 'auto', 'leaf_size': 30, 'metric...   \n",
      "8   {'algorithm': 'auto', 'leaf_size': 30, 'metric...   \n",
      "9   {'algorithm': 'auto', 'leaf_size': 30, 'metric...   \n",
      "10  {'class_weight': None, 'criterion': 'gini', 'm...   \n",
      "11  {'class_weight': None, 'criterion': 'gini', 'm...   \n",
      "12  {'class_weight': None, 'criterion': 'gini', 'm...   \n",
      "13  {'class_weight': None, 'criterion': 'gini', 'm...   \n",
      "14  {'class_weight': None, 'criterion': 'gini', 'm...   \n",
      "15  {'algorithm': 'auto', 'leaf_size': 30, 'metric...   \n",
      "16  {'algorithm': 'auto', 'leaf_size': 30, 'metric...   \n",
      "17  {'algorithm': 'auto', 'leaf_size': 30, 'metric...   \n",
      "18  {'algorithm': 'auto', 'leaf_size': 30, 'metric...   \n",
      "19  {'algorithm': 'auto', 'leaf_size': 30, 'metric...   \n",
      "\n",
      "                        classifier_parameters_p_top20  \\\n",
      "0   {'class_weight': None, 'criterion': 'gini', 'm...   \n",
      "1   {'class_weight': None, 'criterion': 'gini', 'm...   \n",
      "2   {'class_weight': None, 'criterion': 'gini', 'm...   \n",
      "3   {'class_weight': None, 'criterion': 'gini', 'm...   \n",
      "4   {'class_weight': None, 'criterion': 'gini', 'm...   \n",
      "5   {'algorithm': 'auto', 'leaf_size': 30, 'metric...   \n",
      "6   {'algorithm': 'auto', 'leaf_size': 30, 'metric...   \n",
      "7   {'algorithm': 'auto', 'leaf_size': 30, 'metric...   \n",
      "8   {'algorithm': 'auto', 'leaf_size': 30, 'metric...   \n",
      "9   {'algorithm': 'auto', 'leaf_size': 30, 'metric...   \n",
      "10  {'class_weight': None, 'criterion': 'gini', 'm...   \n",
      "11  {'class_weight': None, 'criterion': 'gini', 'm...   \n",
      "12  {'class_weight': None, 'criterion': 'gini', 'm...   \n",
      "13  {'class_weight': None, 'criterion': 'gini', 'm...   \n",
      "14  {'class_weight': None, 'criterion': 'gini', 'm...   \n",
      "15  {'algorithm': 'auto', 'leaf_size': 30, 'metric...   \n",
      "16  {'algorithm': 'auto', 'leaf_size': 30, 'metric...   \n",
      "17  {'algorithm': 'auto', 'leaf_size': 30, 'metric...   \n",
      "18  {'algorithm': 'auto', 'leaf_size': 30, 'metric...   \n",
      "19  {'algorithm': 'auto', 'leaf_size': 30, 'metric...   \n",
      "\n",
      "                          classifier_parameters_brier  \\\n",
      "0   {'class_weight': None, 'criterion': 'gini', 'm...   \n",
      "1   {'class_weight': None, 'criterion': 'gini', 'm...   \n",
      "2   {'class_weight': None, 'criterion': 'gini', 'm...   \n",
      "3   {'class_weight': None, 'criterion': 'gini', 'm...   \n",
      "4   {'class_weight': None, 'criterion': 'gini', 'm...   \n",
      "5   {'algorithm': 'auto', 'leaf_size': 30, 'metric...   \n",
      "6   {'algorithm': 'auto', 'leaf_size': 30, 'metric...   \n",
      "7   {'algorithm': 'auto', 'leaf_size': 30, 'metric...   \n",
      "8   {'algorithm': 'auto', 'leaf_size': 30, 'metric...   \n",
      "9   {'algorithm': 'auto', 'leaf_size': 30, 'metric...   \n",
      "10  {'class_weight': None, 'criterion': 'gini', 'm...   \n",
      "11  {'class_weight': None, 'criterion': 'gini', 'm...   \n",
      "12  {'class_weight': None, 'criterion': 'gini', 'm...   \n",
      "13  {'class_weight': None, 'criterion': 'gini', 'm...   \n",
      "14  {'class_weight': None, 'criterion': 'gini', 'm...   \n",
      "15  {'algorithm': 'auto', 'leaf_size': 30, 'metric...   \n",
      "16  {'algorithm': 'auto', 'leaf_size': 30, 'metric...   \n",
      "17  {'algorithm': 'auto', 'leaf_size': 30, 'metric...   \n",
      "18  {'algorithm': 'auto', 'leaf_size': 30, 'metric...   \n",
      "19  {'algorithm': 'auto', 'leaf_size': 30, 'metric...   \n",
      "\n",
      "                    sampler_parameters_auc  \\\n",
      "0   {'proportion': 0.1, 'n_components': 3}   \n",
      "1                      {'proportion': 1.0}   \n",
      "2                     {'proportion': 0.75}   \n",
      "3              {'proportion': 1.5, 'k': 5}   \n",
      "4     {'n_neighbors': 7, 'threshold': 0.5}   \n",
      "5   {'proportion': 1.0, 'n_components': 4}   \n",
      "6                     {'proportion': 0.75}   \n",
      "7                      {'proportion': 2.0}   \n",
      "8              {'proportion': 1.5, 'k': 5}   \n",
      "9     {'n_neighbors': 7, 'threshold': 0.5}   \n",
      "10  {'proportion': 1.5, 'n_components': 3}   \n",
      "11                     {'proportion': 1.0}   \n",
      "12                     {'proportion': 2.0}   \n",
      "13             {'proportion': 2.0, 'k': 3}   \n",
      "14    {'n_neighbors': 3, 'threshold': 0.3}   \n",
      "15  {'proportion': 1.5, 'n_components': 2}   \n",
      "16                    {'proportion': 0.25}   \n",
      "17                     {'proportion': 1.0}   \n",
      "18             {'proportion': 0.1, 'k': 7}   \n",
      "19    {'n_neighbors': 5, 'threshold': 0.5}   \n",
      "\n",
      "                    sampler_parameters_acc  \\\n",
      "0   {'proportion': 0.1, 'n_components': 3}   \n",
      "1                     {'proportion': 0.75}   \n",
      "2                     {'proportion': 0.25}   \n",
      "3              {'proportion': 1.5, 'k': 5}   \n",
      "4     {'n_neighbors': 5, 'threshold': 0.3}   \n",
      "5   {'proportion': 0.1, 'n_components': 2}   \n",
      "6                      {'proportion': 0.1}   \n",
      "7                     {'proportion': 0.25}   \n",
      "8              {'proportion': 0.1, 'k': 3}   \n",
      "9     {'n_neighbors': 5, 'threshold': 0.3}   \n",
      "10  {'proportion': 1.5, 'n_components': 3}   \n",
      "11                     {'proportion': 1.0}   \n",
      "12                    {'proportion': 0.75}   \n",
      "13             {'proportion': 0.1, 'k': 3}   \n",
      "14    {'n_neighbors': 5, 'threshold': 0.3}   \n",
      "15  {'proportion': 0.1, 'n_components': 2}   \n",
      "16                     {'proportion': 0.1}   \n",
      "17                    {'proportion': 0.75}   \n",
      "18             {'proportion': 0.1, 'k': 7}   \n",
      "19    {'n_neighbors': 5, 'threshold': 0.3}   \n",
      "\n",
      "                    sampler_parameters_gacc  \\\n",
      "0    {'proportion': 0.1, 'n_components': 3}   \n",
      "1                       {'proportion': 1.0}   \n",
      "2                      {'proportion': 0.75}   \n",
      "3               {'proportion': 1.5, 'k': 5}   \n",
      "4      {'n_neighbors': 7, 'threshold': 0.5}   \n",
      "5   {'proportion': 0.25, 'n_components': 2}   \n",
      "6                       {'proportion': 0.1}   \n",
      "7                      {'proportion': 0.75}   \n",
      "8               {'proportion': 1.0, 'k': 3}   \n",
      "9      {'n_neighbors': 5, 'threshold': 0.3}   \n",
      "10   {'proportion': 1.5, 'n_components': 5}   \n",
      "11                      {'proportion': 1.0}   \n",
      "12                      {'proportion': 2.0}   \n",
      "13              {'proportion': 2.0, 'k': 3}   \n",
      "14     {'n_neighbors': 7, 'threshold': 0.5}   \n",
      "15   {'proportion': 1.5, 'n_components': 2}   \n",
      "16                      {'proportion': 1.0}   \n",
      "17                      {'proportion': 1.0}   \n",
      "18              {'proportion': 1.5, 'k': 7}   \n",
      "19     {'n_neighbors': 3, 'threshold': 0.7}   \n",
      "\n",
      "                      sampler_parameters_f1  \\\n",
      "0    {'proportion': 0.1, 'n_components': 3}   \n",
      "1                      {'proportion': 0.75}   \n",
      "2                      {'proportion': 0.75}   \n",
      "3               {'proportion': 1.5, 'k': 5}   \n",
      "4      {'n_neighbors': 5, 'threshold': 0.3}   \n",
      "5   {'proportion': 0.25, 'n_components': 2}   \n",
      "6                       {'proportion': 0.1}   \n",
      "7                      {'proportion': 0.75}   \n",
      "8               {'proportion': 1.0, 'k': 3}   \n",
      "9      {'n_neighbors': 5, 'threshold': 0.3}   \n",
      "10   {'proportion': 1.5, 'n_components': 5}   \n",
      "11                      {'proportion': 1.0}   \n",
      "12                      {'proportion': 2.0}   \n",
      "13              {'proportion': 2.0, 'k': 3}   \n",
      "14     {'n_neighbors': 7, 'threshold': 0.5}   \n",
      "15   {'proportion': 1.5, 'n_components': 2}   \n",
      "16                      {'proportion': 1.5}   \n",
      "17                      {'proportion': 1.0}   \n",
      "18              {'proportion': 1.5, 'k': 7}   \n",
      "19     {'n_neighbors': 3, 'threshold': 0.7}   \n",
      "\n",
      "                sampler_parameters_p_top20  \\\n",
      "0   {'proportion': 0.1, 'n_components': 3}   \n",
      "1                     {'proportion': 0.25}   \n",
      "2                      {'proportion': 1.5}   \n",
      "3              {'proportion': 1.0, 'k': 7}   \n",
      "4     {'n_neighbors': 5, 'threshold': 0.3}   \n",
      "5   {'proportion': 2.0, 'n_components': 3}   \n",
      "6                     {'proportion': 0.75}   \n",
      "7                      {'proportion': 1.0}   \n",
      "8              {'proportion': 0.5, 'k': 7}   \n",
      "9     {'n_neighbors': 5, 'threshold': 0.3}   \n",
      "10  {'proportion': 0.1, 'n_components': 3}   \n",
      "11                     {'proportion': 1.0}   \n",
      "12                     {'proportion': 0.5}   \n",
      "13             {'proportion': 0.1, 'k': 3}   \n",
      "14    {'n_neighbors': 5, 'threshold': 0.3}   \n",
      "15  {'proportion': 1.5, 'n_components': 2}   \n",
      "16                    {'proportion': 0.25}   \n",
      "17                     {'proportion': 1.0}   \n",
      "18             {'proportion': 0.1, 'k': 7}   \n",
      "19    {'n_neighbors': 7, 'threshold': 0.3}   \n",
      "\n",
      "                  sampler_parameters_brier  \n",
      "0   {'proportion': 2.0, 'n_components': 5}  \n",
      "1                      {'proportion': 2.0}  \n",
      "2                      {'proportion': 2.0}  \n",
      "3              {'proportion': 1.5, 'k': 3}  \n",
      "4     {'n_neighbors': 7, 'threshold': 0.7}  \n",
      "5   {'proportion': 0.1, 'n_components': 5}  \n",
      "6                      {'proportion': 2.0}  \n",
      "7                      {'proportion': 0.5}  \n",
      "8              {'proportion': 0.1, 'k': 7}  \n",
      "9     {'n_neighbors': 7, 'threshold': 0.7}  \n",
      "10  {'proportion': 2.0, 'n_components': 5}  \n",
      "11                     {'proportion': 2.0}  \n",
      "12                     {'proportion': 1.5}  \n",
      "13             {'proportion': 0.1, 'k': 7}  \n",
      "14    {'n_neighbors': 5, 'threshold': 0.7}  \n",
      "15  {'proportion': 1.0, 'n_components': 5}  \n",
      "16                     {'proportion': 1.5}  \n",
      "17                     {'proportion': 2.0}  \n",
      "18             {'proportion': 2.0, 'k': 3}  \n",
      "19    {'n_neighbors': 5, 'threshold': 0.7}  \n",
      "\n",
      "[20 rows x 26 columns]\n"
     ]
    }
   ],
   "source": [
    "# The results can be processed according to the requirements of the analysis\n",
    "\n",
    "print(results)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
